# Embeded.js d3 and crossfilter.js in Jupyter Notebook

## Import data

# Read in data
import pandas as pd
f = open("/home/dyfrigmon/Data/FlightData/flights-3m.json")
text=f.readlines()
f.close()

# Get column names
columnNames = text.pop(0)
columnNames = columnNames.replace('\n','').split(',')
print columnNames

['date', 'delay', 'distance', 'origin', 'destination']

# Create DataFrame
df=pd.DataFrame(text)
splitFunc = lambda x: pd.Series([i for i in (x.replace('\n','').split(','))])
df = df[0].apply(splitFunc)
df.columns = columnNames

df.tail()

# Starnge Date Format - think the year has been choped off

from datetime import datetime

dateConvFunc = lambda x: datetime(2001,int(x[0:2]),int(x[2:4]),int(x[4:6]),int(x[6:8]))
df['date']=df['date'].apply(dateConvFunc)

df['index'] = df.index

len(pd.unique(df['origin']))

59

df.dtypes

date           datetime64[ns]
delay                  object
distance               object
origin                 object
destination            object
index                   int64
dtype: object

df['delay'] = df['delay'].astype(int)

df['test'] = (df['delay'] > 5)*1

df2=df

## hash function to convert origin and destination to ints.
df2['origin'].unique()

array(['MCI', 'LAX', 'ONT', 'OAK', 'MSY', 'LAS', 'MDW', 'RNO', 'FLL',
       'SEA', 'ELP', 'SMF', 'BNA', 'SLC', 'TPA', 'BOI', 'SJC', 'ISP',
       'BWI', 'PHX', 'BHM', 'DAL', 'HOU', 'ABQ', 'BDL', 'OKC', 'SDF',
       'MHT', 'LBB', 'PDX', 'OMA', 'AUS', 'BUR', 'MCO', 'CLE', 'DTW',
       'RDU', 'JAX', 'PVD', 'SAN', 'STL', 'IND', 'TUL', 'SFO', 'GEG',
       'MAF', 'SAT', 'CMH', 'ALB', 'BUF', 'TUS', 'AMA', 'JAN', 'HRL',
       'SNA', 'IAH', 'LIT', 'CRP', 'PBI'], dtype=object)

uniqueValues = df2['origin'].unique()
uniqueValues = sorted(uniqueValues)

column = 'origin'
x = 'MCI'
def HashOrigin(x):
    for i, value in enumerate(uniqueValues):
        if x == value:
            return i

df2['test'] = df2['origin'].apply(HashOrigin)

df2

from IPython.display import Javascript
# Bind data to window to allow javascript to access it
Javascript("""
           window.flights={};
           """.format(df2.to_json()))

%%javascript
// Load in the required javascripts
require.config({
    paths: {
        d3: 'https://cdnjs.cloudflare.com/ajax/libs/d3/3.4.8/d3.min',
        crossfilter: 'https://cdnjs.cloudflare.com/ajax/libs/crossfilter/1.3.12/crossfilter.min'
    },
      shim: {
    'crossfilter': {
      deps: [],
      exports: 'crossfilter'
    }
  } 
});

%%javascript
// function which converts 
function convertPlotData(data){
  var convertedData = [];
  for(var i in data[Object.keys(data)[0]]){
    var convertedDatum = {};
    for(var key in data){
      convertedDatum[key] = data[key][i];
    }
    convertedData.push(convertedDatum);
  }
  return convertedData;
}

window.convertPlotData = convertPlotData;

%%javascript
// Test that d3 and crossfilter were imported correctly and that the flight data can be accesed by javascript
// Check console to see results (right click and inspect element)
require(['d3','crossfilter'], function(d3,crossfilter){
    console.log('Tets d3')
    console.log(d3);
    console.log('Test crossfilter')
    console.log(crossfilter)
    console.log('Test data access')
    console.log(flights)
})

# Import style sheet to change the look and feel of plots
from IPython.core.display import HTML
#HTML("""<link href="flightdata.css" rel="stylesheet">""")
HTML("""<style>
@import url(http://fonts.googleapis.com/css?family=Yanone+Kaffeesatz:400,700);

#body > p {
  line-height: 1.5em;
  width: 640px;
  text-rendering: optimizeLegibility;
}

#charts {
  padding: 10px 0;
}

.chart {
  display: inline-block;
  height: 151px;
  margin-bottom: 20px;
}

.reset {
  padding-left: 1em;
  font-size: smaller;
  color: #ccc;
}

.background.bar {
  fill: #ccc;
}

.foreground.bar {
  fill: steelblue;
}

.axis path, .axis line {
  fill: none;
  stroke: #000;
  shape-rendering: crispEdges;
}

.axis text {
  font: 10px sans-serif;
}

.brush rect.extent {
  fill: steelblue;
  fill-opacity: .125;
}

.brush .resize path {
  fill: #eee;
  stroke: #666;
}

#hour-chart {
  width: 260px;
}

#delay-chart {
  width: 230px;
}

#distance-chart {
  width: 420px;
}

#date-chart {
  width: 920px;
}

#flight-list {
  min-height: 1024px;
}

#flight-list .date,
#flight-list .day {
  margin-bottom: .4em;
}

#flight-list .flight {
  line-height: 1.5em;
  background: #eee;
  width: 640px;
  margin-bottom: 1px;
}

#flight-list .time {
  color: #999;
}

#flight-list .flight div {
  display: inline-block;
  width: 100px;
}

#flight-list div.distance,
#flight-list div.delay {
  width: 160px;
  padding-right: 10px;
  text-align: right;
}

#flight-list .early {
  color: green;
}

aside {
  position: absolute;
  left: 740px;
  font-size: smaller;
  width: 220px;
}

</style>
""")

# Create charts
HTML("""<div id="charts">
  <div id="hour-chart" class="chart">
    <div class="title">Time of Day</div>
  </div>
  <div id="delay-chart" class="chart">
    <div class="title">Arrival Delay (min.)</div>
  </div>
  <div id="distance-chart" class="chart">
    <div class="title">Distance (mi.)</div>
  </div>
  <div id="date-chart" class="chart">
    <div class="title">Date</div>
  </div>
  <div id="flight-chart" class="chart">
    <div class="title">Flight Code</div>
  </div>
</div>

<aside id="totals"><span id="active">-</span> of <span id="total">-</span> flights selected.</aside>

<div id="lists">
  <div id="flight-list" class="list"></div>
</div>


""")

%%javascript
// javascript forked form square.github.io/crossfilter
require(['d3','crossfilter'], function(d3,crossfilter){
  var flights = convertPlotData(window.flights);
  // Various formatters.
  var formatNumber = d3.format(",d"),
      formatChange = d3.format("+,d"),
      formatDate = d3.time.format("%B %d, %Y"),
      formatTime = d3.time.format("%I:%M %p");

  // A nest operator, for grouping the flight list.
  var nestByDate = d3.nest()
      .key(function(d) { return d3.time.day(d.date); });

  // A little coercion, since the CSV is untyped.
  flights.forEach(function(d, i) {
    d.index = i;
    d.date = parseDate(d.date);
    d.delay = +d.delay;
    d.distance = +d.distance;
    d.test = d.test
  });

  // Create the crossfilter for the relevant dimensions and groups.
  var flight = crossfilter(flights),
      all = flight.groupAll(),
      date = flight.dimension(function(d) { return d.date; }),
      dates = date.group(d3.time.day),
      hour = flight.dimension(function(d) { return d.date.getHours() + d.date.getMinutes() / 60; }),
      hours = hour.group(Math.floor),
      delay = flight.dimension(function(d) { return Math.max(-60, Math.min(149, d.delay)); }),
      delays = delay.group(function(d) { return Math.floor(d / 10) * 10; }),
      distance = flight.dimension(function(d) { return Math.min(1999, d.distance); }),
      distances = distance.group(function(d) { return Math.floor(d / 50) * 50; }),
      test = flight.dimension(function(d) { return  d.test; }),
      tests = test.group(function(d) { return d; });
  
  var charts = [

    barChart()
        .dimension(hour)
        .group(hours)
      .x(d3.scale.linear()
        .domain([0, 24])
        .rangeRound([0, 10 * 24])),

    barChart()
        .dimension(delay)
        .group(delays)
      .x(d3.scale.linear()
        .domain([-60, 150])
        .rangeRound([0, 10 * 21])),

    barChart()
        .dimension(distance)
        .group(distances)
      .x(d3.scale.linear()
        .domain([0, 2000])
        .rangeRound([0, 10 * 40])),

    barChart()
        .dimension(date)
        .group(dates)
        .round(d3.time.day.round)
      .x(d3.time.scale()
        .domain([new Date(2001, 0, 1), new Date(2001, 3, 1)])
        .rangeRound([0, 10 * 90]))
        .filter([new Date(2001, 1, 1), new Date(2001, 2, 1)]),
     
    barChart()
        .dimension(test)
        .group(tests)
      .x(d3.scale.ordinal()
        .domain(['ABQ','AUS','BDL','BHM','BNA','BOI','BUR','BWI','CLE','DAL',
 'DTW','ELP','FLL','GEG','HOU','IND','ISP','JAX','LAS','LAX','LBB','MCI','MCO','MDW','MHT','MSY',
 'OAK','OKC','OMA','ONT','PDX','PHX','PVD','RDU','RNO','SAN','SDF','SEA',
 'SFO','SJC','SLC','SMF','STL','TPA','TUL'])
        .rangeBands([0,
 1,
 2,
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
 11,
 12,
 13,
 14,
 15,
 16,
 17,
 18,
 19,
 20,
 21,
 22,
 23,
 24,
 25,
 26,
 27,
 28,
 29,
 30,
 31,
 32,
 33,
 34,
 35,
 36,
 37,
 38,
 39,
 40,
 41,
 42,
 43,
 44]))
      
  ]
  
  //temp = d3.select('#gender-chart').selectAll('.axis').selectAll('text')[0][0]
  //d3.select(temp).text('Male').attr('transform', 'rotate(-45) translate(-12,-2)');
       
//    d3.select('#flight-chart').axis()
  //      .scale(x).tickValues(["Year 1", "year 2"])

  // Given our array of charts, which we assume are in the same order as the
  // .chart elements in the DOM, bind the charts to the DOM and render them.
  // We also listen to the chart's brush events to update the display.
  var chart = d3.selectAll(".chart")
      .data(charts)
      .each(function(chart) { chart.on("brush", renderAll).on("brushend", renderAll); });

  // Render the initial lists.
  var list = d3.selectAll(".list")
      .data([flightList]);

  // Render the total.
  d3.selectAll("#total")
      .text(formatNumber(flight.size()));

  renderAll();

  // Renders the specified chart or list.
  function render(method) {
    d3.select(this).call(method);
  }

  // Whenever the brush moves, re-rendering everything.
  function renderAll() {
    chart.each(render);
    list.each(render);
    d3.select("#active").text(formatNumber(all.value()));
  }

  // Convert Unix Time to d3
  function parseDate(d) {
    return new Date(d)}

  window.filter = function(filters) {
    filters.forEach(function(d, i) { charts[i].filter(d); });
    renderAll();
  };

  window.reset = function(i) {
    charts[i].filter(null);
    renderAll();
  };

  function flightList(div) {
    var flightsByDate = nestByDate.entries(date.top(40));

    div.each(function() {
      var date = d3.select(this).selectAll(".date")
          .data(flightsByDate, function(d) { return d.key; });

      date.enter().append("div")
          .attr("class", "date")
        .append("div")
          .attr("class", "day")
          .text(function(d) { return formatDate(d.values[0].date); });

      date.exit().remove();

      var flight = date.order().selectAll(".flight")
          .data(function(d) { return d.values; }, function(d) { return d.index; });

      var flightEnter = flight.enter().append("div")
          .attr("class", "flight");

      flightEnter.append("div")
          .attr("class", "time")
          .text(function(d) { return formatTime(d.date); });

      flightEnter.append("div")
          .attr("class", "origin")
          .text(function(d) { return d.origin; });

      flightEnter.append("div")
          .attr("class", "destination")
          .text(function(d) { return d.destination; });

      flightEnter.append("div")
          .attr("class", "distance")
          .text(function(d) { return formatNumber(d.distance) + " mi."; });

      flightEnter.append("div")
          .attr("class", "delay")
          .classed("early", function(d) { return d.delay < 0; })
          .text(function(d) { return formatChange(d.delay) + " min."; });

      flight.exit().remove();

      flight.order();
    });
  }

    
    
    
    
    
    
/// Don't touch stuff below here.    
    
  function barChart() {
    if (!barChart.id) barChart.id = 0;

    var margin = {top: 10, right: 10, bottom: 20, left: 10},
        x,
        y = d3.scale.linear().range([100, 0]),
        id = barChart.id++,
        axis = d3.svg.axis().orient("bottom"),
        brush = d3.svg.brush(),
        brushDirty,
        dimension,
        group,
        round;
      // add y axis here like axis

    function chart(div) {
      var width = x.range()[1],
          height = y.range()[0];

      y.domain([0, group.top(1)[0].value]);

      div.each(function() {
        var div = d3.select(this),
            g = div.select("g");

        // Create the skeletal chart.
        if (g.empty()) {
          div.select(".title").append("a")
              .attr("href", "javascript:reset(" + id + ")")
              .attr("class", "reset")
              .text("reset")
              .style("display", "none");

          g = div.append("svg")
              .attr("width", width + margin.left + margin.right)
              .attr("height", height + margin.top + margin.bottom)
            .append("g")
              .attr("transform", "translate(" + margin.left + "," + margin.top + ")");

          g.append("clipPath")
              .attr("id", "clip-" + id)
            .append("rect")
              .attr("width", width)
              .attr("height", height);

          g.selectAll(".bar")
              .data(["background", "foreground"])
            .enter().append("path")
              .attr("class", function(d) { return d + " bar"; })
              .datum(group.all());

          g.selectAll(".foreground.bar")
              .attr("clip-path", "url(#clip-" + id + ")");

          g.append("g")
              .attr("class", "axis")
              .attr("transform", "translate(0," + height + ")")
              .call(axis);
            
        //add in y axis here

          // Initialize the brush component with pretty resize handles.
          var gBrush = g.append("g").attr("class", "brush").call(brush);
          gBrush.selectAll("rect").attr("height", height);
          gBrush.selectAll(".resize").append("path").attr("d", resizePath);
        }

        // Only redraw the brush if set externally.
        if (brushDirty) {
          brushDirty = false;
          g.selectAll(".brush").call(brush);
          div.select(".title a").style("display", brush.empty() ? "none" : null);
          if (brush.empty()) {
            g.selectAll("#clip-" + id + " rect")
                .attr("x", 0)
                .attr("width", width);
          } else {
            var extent = brush.extent();
            g.selectAll("#clip-" + id + " rect")
                .attr("x", x(extent[0]))
                .attr("width", x(extent[1]) - x(extent[0]));
          }
        }

        g.selectAll(".bar").attr("d", barPath);
      });

      function barPath(groups) {
        var path = [],
            i = -1,
            n = groups.length,
            d;
        while (++i < n) {
          d = groups[i];
          path.push("M", x(d.key), ",", height, "V", y(d.value), "h9V", height);
        }
        return path.join("");
      }

      function resizePath(d) {
        var e = +(d == "e"),
            x = e ? 1 : -1,
            y = height / 3;
        return "M" + (.5 * x) + "," + y
            + "A6,6 0 0 " + e + " " + (6.5 * x) + "," + (y + 6)
            + "V" + (2 * y - 6)
            + "A6,6 0 0 " + e + " " + (.5 * x) + "," + (2 * y)
            + "Z"
            + "M" + (2.5 * x) + "," + (y + 8)
            + "V" + (2 * y - 8)
            + "M" + (4.5 * x) + "," + (y + 8)
            + "V" + (2 * y - 8);
      }
    }

    brush.on("brushstart.chart", function() {
      var div = d3.select(this.parentNode.parentNode.parentNode);
      div.select(".title a").style("display", null);
    });

    brush.on("brush.chart", function() {
      var g = d3.select(this.parentNode),
          extent = brush.extent();
      if (round) g.select(".brush")
          .call(brush.extent(extent = extent.map(round)))
        .selectAll(".resize")
          .style("display", null);
      g.select("#clip-" + id + " rect")
          .attr("x", x(extent[0]))
          .attr("width", x(extent[1]) - x(extent[0]));
      dimension.filterRange(extent);
    });

    brush.on("brushend.chart", function() {
      if (brush.empty()) {
        var div = d3.select(this.parentNode.parentNode.parentNode);
        div.select(".title a").style("display", "none");
        div.select("#clip-" + id + " rect").attr("x", null).attr("width", "100%");
        dimension.filterAll();
      }
    });

    chart.margin = function(_) {
      if (!arguments.length) return margin;
      margin = _;
      return chart;
    };

    chart.x = function(_) {
      if (!arguments.length) return x;
      x = _;
      axis.scale(x);
      brush.x(x);
      return chart;
    };

    chart.y = function(_) {
      if (!arguments.length) return y;
      y = _;
      return chart;
    };

    chart.dimension = function(_) {
      if (!arguments.length) return dimension;
      dimension = _;
      return chart;
    };

    chart.filter = function(_) {
      if (_) {
        brush.extent(_);
        dimension.filterRange(_);
      } else {
        brush.clear();
        dimension.filterAll();
      }
      brushDirty = true;
      return chart;
    };

    chart.group = function(_) {
      if (!arguments.length) return group;
      group = _;
      return chart;
    };

    chart.round = function(_) {
      if (!arguments.length) return round;
      round = _;
      return chart;
    };

    return d3.rebind(chart, brush, "on");
  }
});

	date	delay	distance	origin	destination
231078	03312210	-7	423	OMA	MDW
231079	03312230	-8	303	HOU	MSY
231080	03312237	106	443	BNA	RDU
231081	03312248	125	880	STL	MCO
231082	03312257	29	303	MSY	HOU

	date	delay	distance	origin	destination	index	test
0	2001-01-01 00:01:00	14	405	MCI	MDW	0	31
1	2001-01-01 05:30:00	-11	370	LAX	PHX	1	27
2	2001-01-01 05:40:00	5	389	ONT	SMF	2	39
3	2001-01-01 06:00:00	-5	337	OAK	LAX	3	36
4	2001-01-01 06:00:00	3	303	MSY	HOU	4	35
5	2001-01-01 06:05:00	5	236	LAS	LAX	5	26
6	2001-01-01 06:10:00	-4	405	MDW	MCI	6	33
7	2001-01-01 06:15:00	-2	188	RNO	SJC	7	45
8	2001-01-01 06:15:00	0	197	FLL	TPA	8	17
9	2001-01-01 06:15:00	0	399	SEA	BOI	9	49
10	2001-01-01 06:15:00	5	562	ELP	DAL	10	16
11	2001-01-01 06:20:00	-5	358	SMF	BUR	11	53
12	2001-01-01 06:20:00	0	491	BNA	MCI	12	6
13	2001-01-01 06:25:00	-6	361	ONT	OAK	13	39
14	2001-01-01 06:25:00	0	313	MCI	OKC	14	31
15	2001-01-01 06:25:00	1	271	MDW	SDF	15	33
16	2001-01-01 06:25:00	5	689	SLC	SEA	16	52
17	2001-01-01 06:30:00	-1	487	TPA	MSY	17	56
18	2001-01-01 06:30:00	-10	399	BOI	SEA	18	7
19	2001-01-01 06:30:00	-15	621	SJC	PHX	19	51
20	2001-01-01 06:30:00	-2	361	OAK	ONT	20	36
21	2001-01-01 06:30:00	-3	220	ISP	BWI	21	23
22	2001-01-01 06:30:00	-3	397	SMF	LAS	22	53
23	2001-01-01 06:30:00	-8	251	MDW	STL	23	33
24	2001-01-01 06:30:00	-8	417	SJC	SAN	24	51
25	2001-01-01 06:30:00	1	682	BWI	BHM	25	10
26	2001-01-01 06:30:00	10	370	PHX	LAX	26	42
27	2001-01-01 06:30:00	15	177	BHM	BNA	27	5
28	2001-01-01 06:30:00	2	562	DAL	ELP	28	14
29	2001-01-01 06:35:00	0	453	HOU	TUL	29	19
30	2001-01-01 06:35:00	1	601	RNO	PHX	30	45
31	2001-01-01 06:35:00	2	670	BNA	HOU	31	6
32	2001-01-01 06:35:00	9	677	ABQ	LAX	32	0
33	2001-01-01 06:40:00	-21	777	BDL	MDW	33	4
34	2001-01-01 06:40:00	-8	197	ONT	LAS	34	39
35	2001-01-01 06:40:00	3	833	OKC	PHX	35	37
36	2001-01-01 06:40:00	5	495	SDF	BWI	36	48
37	2001-01-01 06:45:00	-14	423	MDW	OMA	37	33
38	2001-01-01 06:45:00	-14	605	SEA	SMF	38	49
39	2001-01-01 06:45:00	-19	838	MHT	MDW	39	34